{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "# Change to the parent directory\n",
    "os.chdir('..')\n",
    "import torch\n",
    "import torchvision\n",
    "import os\n",
    "from os.path import join as j_\n",
    "from PIL import Image\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# loading all packages here to start\n",
    "from panderm_model import get_encoder\n",
    "from panderm_model.downstream.extract_features import extract_features_from_dataloader\n",
    "from panderm_model.downstream.eval_features.linear_probe import eval_linear_probe\n",
    "from panderm_model.downstream.eval_features.metrics import get_eval_metrics, print_metrics\n",
    "\n",
    "from datasets.derm_data import Derm_Dataset\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Downloading Model weights + Creating Model\n",
    "\n",
    "Download the pre-trained model weights from [this Google Drive link](https://drive.google.com/file/d/1XHKRk2p-dS1PFQE-xRbOM3yx47i3bXmi/view?usp=sharing).\n",
    "\n",
    "### Configuring the Model Path\r\n",
    "After downloading, you need to update the model weights path in the code:\r\n",
    "\r\n",
    "1. Open the file `PanDerm/LP_Eval/models/builder.py`\r\n",
    "2. Locate line 42\r\n",
    "3. Replace the existing path with the directory where you saved the model weights:\r\n",
    "\r\n",
    "```python\r\n",
    "root_path = '/path/to/your/PanDerm/Model_Weights/'\r\n",
    "```\n",
    "\n",
    "Set model_name to one of 'PanDerm' 'SwAVDerm'  'dinov2' 'imgnet_large21k', which is our model and the main comparative models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading model checkpoint\n",
      "VisionTransformer(\n",
      "  (patch_embed): PatchEmbed(\n",
      "    (proj): Conv2d(3, 1024, kernel_size=(16, 16), stride=(16, 16))\n",
      "  )\n",
      "  (pos_drop): Dropout(p=0.0, inplace=False)\n",
      "  (blocks): ModuleList(\n",
      "    (0): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): Identity()\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (1): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.004347826354205608)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (2): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.008695652708411217)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (3): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.013043479062616825)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (4): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.017391305416822433)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (5): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.021739132702350616)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (6): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.02608695812523365)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (7): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.030434783548116684)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (8): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.03478261083364487)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (9): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.03913043811917305)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (10): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.04347826540470123)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (11): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.04782608896493912)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (12): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.052173912525177)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (13): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.056521736085414886)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (14): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.06086956337094307)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (15): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.06521739065647125)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (16): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.06956521421670914)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (17): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.07391304522752762)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (18): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.0782608687877655)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (19): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.08260869979858398)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (20): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.08695652335882187)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (21): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.09130434691905975)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (22): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.09565217792987823)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (23): Block(\n",
      "      (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (attn): Attention(\n",
      "        (qkv): Linear(in_features=1024, out_features=3072, bias=False)\n",
      "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "        (proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "      (drop_path): DropPath(p=0.10000000149011612)\n",
      "      (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "      (mlp): Mlp(\n",
      "        (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "        (act): GELU(approximate='none')\n",
      "        (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (norm): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
      "  (head): Linear(in_features=1024, out_features=1000, bias=True)\n",
      ")\n",
      "normalization method:  imagenet\n"
     ]
    }
   ],
   "source": [
    "from models import get_encoder\n",
    "model_name='PanDerm'\n",
    "model, eval_transform = get_encoder(model_name)\n",
    "_ = model.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare Evaluation Dataset (HAM10000 as an example) and Important Hyperparameters\r\n",
    "\r\n",
    "#### Dataset Setup\r\n",
    "1. Download the provided [dataset](https://drive.google.com/file/d/1D9Q4B50Z5tyj5fd5EE9QWmFrg66vGvfA/view).\r\n",
    "2. Update the following key paths:\r\n",
    "   - `csv_path`: Path to the CSV file containing data paths and labels\r\n",
    "   - `root_path`: Directory containing the image files\r\n",
    "   - `output_dir`: Directory where results will be saved\r\n",
    "\r\n",
    "#### Key Parameters\r\n",
    "Adjust these important parameters:\r\n",
    "\r\n",
    "- `nb_classes`: Set this to the number of classes in your evaluation dataset\r\n",
    "- `batch_size`: Adjust based on the memory size of your GPU\r\n",
    "- `percent_data`: Controls the percentage of training data used\r\n",
    "  - Example: 0.1 means evaluate models using 10% training data\r\n",
    "  - Modify this for label efficiency generalization experimentseriments.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_path=\"/home/share/Uni_Eval/HAM10000_clean/ISIC2018_splits/HAM_clean.csv\" \n",
    "root_path=\"/home/share/Uni_Eval/HAM10000_clean/ISIC2018/\"\n",
    "output_dir='/home/share/FM_Code/PanDerm/LP_Eval/output_dir/Task1/'+model_name\n",
    "# set the class number of your evaluation dataset\n",
    "nb_classes=7\n",
    "model = model.to(device)\n",
    "if nb_classes == 2:\n",
    "    binary = True\n",
    "else:\n",
    "    binary = False\n",
    "\n",
    "batch_size=1000\n",
    "num_workers=8\n",
    "percent_data=1.0\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train size: 8207 ,val size: 575 ,test size: 1232\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(csv_path)\n",
    "dataset_train = Derm_Dataset(df=df,\n",
    "                            root=root_path,\n",
    "                            train=True,\n",
    "                            transforms=eval_transform,\n",
    "                            binary=binary,\n",
    "                            data_percent=percent_data)\n",
    "dataset_val = Derm_Dataset(df=df,\n",
    "                          root=root_path,\n",
    "                          val=True,\n",
    "                          transforms=eval_transform,\n",
    "                          binary=binary)\n",
    "dataset_test = Derm_Dataset(df=df,\n",
    "                           root=root_path,\n",
    "                           test=True,\n",
    "                           transforms=eval_transform,\n",
    "                           binary=binary)\n",
    "print('train size:', len(dataset_train), ',val size:', len(dataset_val), ',test size:', len(dataset_test))\n",
    "\n",
    "\n",
    "import time\n",
    "from panderm_model.downstream.extract_features import extract_features_from_dataloader\n",
    "\n",
    "train_dataloader = torch.utils.data.DataLoader(\n",
    "    dataset_train,\n",
    "    batch_size=batch_size,\n",
    "    shuffle=False,\n",
    "    num_workers=num_workers,\n",
    "    pin_memory=True\n",
    ")\n",
    "\n",
    "val_dataloader = torch.utils.data.DataLoader(\n",
    "    dataset_val,\n",
    "    batch_size=batch_size,\n",
    "    shuffle=False,\n",
    "    num_workers=num_workers,\n",
    "    pin_memory=True\n",
    ")\n",
    "test_dataloader = torch.utils.data.DataLoader(\n",
    "    dataset_test,\n",
    "    batch_size=batch_size,\n",
    "    shuffle=False,\n",
    "    num_workers=num_workers,\n",
    "    pin_memory=True\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 9/9 [01:16<00:00,  8.50s/it]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:14<00:00, 14.18s/it]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:26<00:00, 13.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Took 119.596 seconds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "# extract features from the train and test datasets (returns dictionary of embeddings and labels)\n",
    "train_features = extract_features_from_dataloader(model, train_dataloader)\n",
    "val_features = extract_features_from_dataloader(model, val_dataloader)\n",
    "test_features = extract_features_from_dataloader(model, test_dataloader)\n",
    "\n",
    "# convert these to torch\n",
    "train_feats = torch.Tensor(train_features['embeddings'])\n",
    "train_labels = torch.Tensor(train_features['labels']).type(torch.long)\n",
    "val_feats = torch.Tensor(val_features['embeddings'])\n",
    "val_labels = torch.Tensor(val_features['labels']).type(torch.long)\n",
    "test_feats = torch.Tensor(test_features['embeddings'])\n",
    "test_labels = torch.Tensor(test_features['labels']).type(torch.long)\n",
    "test_filenames = test_features['filenames']\n",
    "elapsed = time.time() - start\n",
    "print(f'Took {elapsed:.03f} seconds')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear Probe Evaluation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation Dataset: HAM_clean\n",
      "Linear Probe Evaluation: Train shape torch.Size([8207, 1024])\n",
      "Linear Probe Evaluation: Test shape torch.Size([1232, 1024])\n",
      "NUM_C, Cost: 7 71.68\n",
      "Linear Probe Evaluation (Train Time): Best cost = 71.680\n",
      "Linear Probe Evaluation (Train Time): Using only train set for training. Train Shape:  torch.Size([8207, 1024])\n",
      "0\n",
      "(Before Training) Loss: 2.023\n",
      "(After Training) Loss: 0.381\n",
      "Linear Probe Evaluation (Test Time): Test Shape torch.Size([1232, 1024])\n",
      "Classification report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.78      0.60      0.68        35\n",
      "           1       0.68      0.82      0.74        44\n",
      "           2       0.82      0.79      0.81       107\n",
      "           3       0.88      0.88      0.88         8\n",
      "           4       0.70      0.63      0.66        70\n",
      "           5       0.97      0.98      0.97       951\n",
      "           6       1.00      0.88      0.94        17\n",
      "\n",
      "    accuracy                           0.92      1232\n",
      "   macro avg       0.83      0.80      0.81      1232\n",
      "weighted avg       0.92      0.92      0.92      1232\n",
      "\n",
      "Debug: targets_all shape: (1232,)\n",
      "Debug: preds_all shape: (1232,)\n",
      "Debug: probs_all shape: (1232, 7)\n",
      "Debug: test_filenames length: 1232\n",
      "Model predicted results saved to /home/share/FM_Code/PanDerm/LP_Eval/output_dir/Task1/PanDerm/HAM_clean.csv\n",
      "Linear Probe Evaluation: Time taken 4.67\n",
      "Test lin_acc: 0.923701\n",
      "Test lin_bacc: 0.796631\n",
      "Test lin_kappa: 0.880742\n",
      "Test lin_weighted_f1: 0.922471\n",
      "Test lin_auroc: 0.977242\n",
      "Test lin_aupr: 0.877725\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x576 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\"linear evaluation\"\"\"\n",
    "from panderm_model.downstream.eval_features.linear_probe import eval_linear_probe\n",
    "dataset_name=str(csv_path).split('/')[-1].split('.')[0]\n",
    "print('Evaluation Dataset:',dataset_name)\n",
    "linprobe_eval_metrics, linprobe_dump = eval_linear_probe(\n",
    "    train_feats=train_feats,\n",
    "    train_labels=train_labels,\n",
    "    valid_feats=None,\n",
    "    valid_labels=val_labels,\n",
    "    test_feats=test_feats,\n",
    "    test_labels=test_labels,\n",
    "    test_filenames=test_filenames,\n",
    "    max_iter=1000,\n",
    "    verbose=True, seed=0,\n",
    "    out_dir=output_dir,\n",
    "    dataset_name=dataset_name\n",
    ")\n",
    "\n",
    "print_metrics(linprobe_eval_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fm_eval",
   "language": "python",
   "name": "fm_eval"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
